/*
 * 
 * Each job crawls a single page and finds all the anchors and links that match a criteria
 */

package jobs;

import java.io.IOException;
import java.net.URI;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.HttpResponse;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.ClientProtocolException;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.InputStream;

import core.Config;
import core.Crawler;
import database.SqlManager;

public class CrawlPageJob extends BaseJob 
{
	URI mUrl;
	private DefaultHttpClient client;
	public static volatile int countJobs = 0;
	
	private int mLastContentLength;
	private String mLastContentType;
	
	public CrawlPageJob( URI pageURL) 
	{
		countJobs++;
		mUrl = pageURL;
		client = new DefaultHttpClient();
		
		if ( mUrl.getScheme() == null )
		{
			mUrl = URI.create("http://" + pageURL.toString());
		}
	}
	
	/* method that gets the page source from url*/
	public String getPageSource( String url )
	{
		String content = "";
		
		try 
		{
			HttpParams params = new BasicHttpParams();
			params.setParameter("http.socket.timeout", 5000);
			params.setParameter("http.connection.timeout", 5000);
			client.setParams(params);
			HttpResponse response = client.execute(new HttpGet(url));			
			
			HttpEntity entity = response.getEntity();
			if (entity != null) 
			{
			    InputStream instream = entity.getContent();

			    BufferedReader reader = new BufferedReader( new InputStreamReader(instream) );

			    String line = reader.readLine();
			    while (line != null)
			    {
			    	content += line;
			    	line = reader.readLine();
			    }
			    
			    
			    return content;
			}
			
			
			return null;
		}
		catch (ClientProtocolException e) 
		{
			return null;
		} 
		catch (IOException e) 
		{
			return null;
		}

			
	}
	/*return true if the image at imageURL fits the crawler critera (statuscode, size) */
	public boolean imageFitsCriteria(URI imageURL)
	{
		try 
		{
			HttpResponse response = client.execute(new HttpGet(imageURL));
			
			if (response.getStatusLine().getStatusCode() != 200)
			{
			    EntityUtils.consume(response.getEntity());
				return false;
			}
			
			Header [] ContentType  = response.getHeaders("Content-Type");
			Header [] ContentLength  = response.getHeaders("Content-Length");
			
			EntityUtils.consume(response.getEntity());
			
			if (ContentType.length == 0 || ContentLength.length == 0)
			{
				return false;
			}
			
			mLastContentType = ContentType[0].getValue();
			mLastContentLength = Integer.parseInt(ContentLength[0].getValue());
		
			switch (ContentType[0].getValue())
			{
				case "image/png":
					if (Integer.parseInt(ContentLength[0].getValue()) > Config.PNG_MIN_SIZE_IN_BYTES)
					{
						return true;
					}
				break;
				
				case "image/gif":
					if (Integer.parseInt(ContentLength[0].getValue()) > Config.GIF_MIN_SIZE_IN_BYTES)
					{
						return true;
					}
				break;
				
				case "image/jpg":
					if (Integer.parseInt(ContentLength[0].getValue()) > Config.JPG_MIN_SIZE_IN_BYTES)
					{
						return true;
					}
				break;
				
				case "image/jpeg":
					if (Integer.parseInt(ContentLength[0].getValue()) > Config.JPG_MIN_SIZE_IN_BYTES)
					{
						return true;
					}
				break;
			}
			
			return false;
		}
		catch (ClientProtocolException e) 
		{
			return false;
		} 
		catch (IOException e) 
		{			
			return false;
		}
			
			
	}

	@Override
	public boolean run() 
	{
		String content = getPageSource( mUrl.toString() );
		
		if (content == null)
		{
			//System.out.print("Content timed out");
			return false;
		}
		
		client = new DefaultHttpClient();
		HttpParams params = new BasicHttpParams();
		params.setParameter("http.socket.timeout", 5000);
		params.setParameter("http.connection.timeout", 5000);
		client.setParams(params);
		
		/* marks the link as visited (temporarily) */
		SqlManager.getInstance().addVisitedLink(mUrl);
		
		Matcher titleMatcher = Pattern.compile("<title>(.*?)</title>").matcher(content);
		Matcher imageMatcher = Pattern.compile("<img[^>]*?src=\"(.*?)\"[^>]*?/?>").matcher(content);
		Matcher anchorMatcher = Pattern.compile("<a[^>]*?href=\"(.*?)\"[^>]*?>").matcher(content);
		
		/* find title */
		String pageTitle = "No title";
		if ( titleMatcher.find() )
		{
			pageTitle = titleMatcher.group(1);
		}
		
		/* find all images */
		while (imageMatcher.find())
		{
			try
			{
				URI imageURI = URI.create(imageMatcher.group(1));
				
				if (imageURI.toString().isEmpty())
				{
					break;
				}
				
				if (isRelative(imageURI))
				{
					imageURI = createAbsoluteFromRelative(mUrl, imageURI);
				}
				
				if (imageFitsCriteria(imageURI))
				{
					/* send them to the image handler */
					Crawler.getInstance().addFoundImageUrl(mUrl, imageURI, pageTitle, mLastContentType, mLastContentLength, Crawler.getInstance().getCurrentWebsite().toString());
				}
			} 
			catch (IllegalArgumentException e)
			{
			}
			
		}
		
		/* find all anchors */
		while (anchorMatcher.find())
		{
			
			//System.out.println("Found " + anchorMatcher.group(1));
			try
			{
				URI anchorURI = URI.create(anchorMatcher.group(1));
				
				if (anchorURI.toString().isEmpty())
				{
					break;
				}
				if (isRelative(anchorURI))
				{
					anchorURI = createAbsoluteFromRelative(mUrl, anchorURI);
					Crawler.getInstance().addFoundPageUrl(anchorURI);
				}
				else
				{
					if (urlFromTheSameSite(anchorURI))
					{
						Crawler.getInstance().addFoundPageUrl(anchorURI);
					}
					else
					{
						Crawler.getInstance().addPotentialSeed(anchorURI);
					}
				}
				
				
			}
			catch (IllegalArgumentException e)
			{
			}
		}
		
		
		return true;
	}
	
	/* returns true if the url has the same scheme and authority as url of the currently crawled page */
	private boolean urlFromTheSameSite(URI url)
	{
		if (!mUrl.getScheme().equals(url.getScheme()))
		{
			return false;
		}
		
		if (!mUrl.getAuthority().equals(url.getAuthority()))
		{
			return false;
		}
		
		return true;
	}
	
	/* return true if it's a relative path */
	private boolean isRelative(URI url)
	{
		if (url.getScheme() == null || url.getAuthority() == null)
		{
			return true;
		}
		
		return false;
	}
	
	/* Returns absolute URI from relative URI */
	private URI createAbsoluteFromRelative(URI url, URI relative)
	{
		String rel = relative.toString();
		if ( rel.charAt(0) == '/' )
		{
			try
			{
				String newUrl = url.getScheme() + "://" + url.getAuthority();
				newUrl += rel;
				
				return URI.create(newUrl);
			}
			catch (IllegalArgumentException e)
			{		
			}
		}
		else
		{
			String abs;
			try 
			{
				abs = url.getPath().substring(0, url.getPath().lastIndexOf('/'));
			}
			catch (StringIndexOutOfBoundsException e)
			{
				abs = url.getPath();
			}
			
			try
			{
				String newUrl = url.getScheme() + "://" + url.getAuthority();
				newUrl += "/" + abs + "/" + relative.toString();
				
				return URI.create(newUrl).normalize();
			}
			catch (IllegalArgumentException e)
			{		
			}
		}
		
		return URI.create("");
	}
}
